package net;

import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import code.extract.FileDealer;

public class GooglePageParserRegexp{

	/**
	 * @param args
	 */
	public static void main(String[] args) {
//		BufferedReader br;
//		try {
//			br = new BufferedReader(new FileReader("google/page/0_0.google.html"));
//			String line = null;
//			String filecontent = "";
//			while ((line = br.readLine()) != null) {
//				filecontent += line + "\r\n";
//			}
//			GooglePageParserRegexp parser = new GooglePageParserRegexp();
//			
////			String s = "<div class=\"med\" style=\"margin-bottom: 2.3em;\"><img src=\"/images/yellow_warning.gif\" alt=\"Information\" height=\"14\" width=\"14\">&nbsp;No results found for <b>\"http://www.professional.ch/_vti_bin/UserGroup.asmx?wsdl\"</b>.</div>";
////			System.out.println(parser.check4NoResults(filecontent));
//			ArrayList<GoogleResultItem> list = parser.parse(filecontent, "WSDL");
//			for (GoogleResultItem googleResultItem : list) {
//				System.out.println("BackLink:\t"
//						+ googleResultItem.getOriginalURL());
//				System.out.println("Anchor:\t"
//						+ googleResultItem.getAnchorText());
//				System.out
//						.println("Summary:\t" + googleResultItem.getSummary());
//			}
//		} catch (FileNotFoundException e) {
//			// TODO Auto-generated catch block
//			e.printStackTrace();
//		} catch (IOException e) {
//			// TODO Auto-generated catch block
//			e.printStackTrace();
//		}
		
		try {
			String content = FileDealer.getFileContent("C:/Users/ftc/Desktop/search_hits.htm");
			GooglePageParserRegexp parser = new GooglePageParserRegexp();
			ArrayList<GoogleResultItem> list = parser.parse(content, "WSDL");
			System.out.println(list.size());
			for (int i=0; i<list.size(); ++i){
				System.out.println(i + ":" + list.get(i).getOriginalURL());
			}
//			System.out.println(content);
//			System.out.println(getResultNumOfQuery(content));
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}
	/**
	 * check whether the returned page has no matched results, but google transforms the query by removing the quotes
	 * <div class="med" style="margin-bottom: 2.3em;"><img src="/images/yellow_warning.gif" alt="Information" height="14" width="14">&nbsp;No results found for <b>"http://www.professional.ch/_vti_bin/UserGroup.asmx?wsdl"</b>.</div>
	 * @param pagecontent
	 * @return false if the page actually has no results
	 * 		   true if the page has results
	 */
	private boolean check4NoResults(String pagecontent){
		Pattern p = Pattern.compile(".+?<img src=\"/images/yellow_warning.gif\".+?No results found for.+?",Pattern.DOTALL);
//		Pattern p = Pattern.compile(".?+<img src=\"/images/yellow_warning\\.gif\".?+");
		Matcher m = p.matcher(pagecontent);
		if(m.matches())
			return false;
		else
			return true;
	}
	public ArrayList<GoogleResultItem> parse(String pagecontent,
			String queryType) {
		ArrayList<GoogleResultItem> lists = new ArrayList<GoogleResultItem>();
		//if there are no results actually in the page
		if(!check4NoResults(pagecontent)){
			return lists;
		}
//		System.out.println(pagecontent);
		/**
		 * Level 1: <li class=g>
		 * <h3 class="r"><a href="http://estagiodissertacao.googlecode.com/svn/trunk/sources/WSScriptGen/DEI_estagio_new_wsdl.txt"
		 * target=_blank class=l
		 * onmousedown="return clk(0,'','','','1','','0CBIQFjAA')">
		 * <em>http</em>://118.sercultur.pt/ws.asmx?<em>WSDL http</em>
		 * ://122.166.2.109 <b>...</b></a></h3><div class="s">
		 * <em>WSDL http</em>://www.ahservizi.it/service.asmx?
		 * <em>WSDL http://www.aironline</em>.<em>pl</em>/<em>server</em>
		 * /<wbr>Rules.<em>php</em>?<em>wsdl</em> <b>......</b>
		 * <em>WSDL http</em>://www.railtour.ch/<em>booking</em>
		 * /GenvasService.asmx? <b>...</b><br>
		 * <span class=f><cite>estagiodissertacao.googlecode.com/svn/.../
		 * DEI_estagio_new_<b>wsdl</b>.txt</cite> - <span class=gl><a href="http://webcache.googleusercontent.com/search?q=cache:durPih_yqcAJ:estagiodissertacao.googlecode.com/svn/trunk/sources/WSScriptGen/DEI_estagio_new_wsdl.txt+http://www.aironline.pl/server/booking.php%3Fwsdl&amp;cd=1&amp;hl=en&amp;ct=clnk"
		 * target=_blank onmousedown="return clk(0,'','','','1','','0CBQQIDAA')"
		 * >Cached</a></span></span></div><!--n-->
		 */
		Pattern pattern_lev1 = Pattern.compile("<li class=g((\\s*?)|(\\s+?.*?))>(.+?)<span class=gl>",
				Pattern.DOTALL);
		Matcher m_level1 = pattern_lev1.matcher(pagecontent);
		if (!m_level1.find()){
			pattern_lev1 = Pattern.compile("<li class=\"g\"((\\s*?)|(\\s+?.*?))>(.+?)<span class=\"gl\">",Pattern.DOTALL);
			 m_level1 = pattern_lev1.matcher(pagecontent);
		}
		int count = 0;
//		int start = 0;
		while (m_level1.find()) {
//			start = m_level1.end()-1;
//			System.out.println(start);
			GoogleResultItem item = new GoogleResultItem();
			item.setRank(count++);
			item.setQueryType(queryType);

			String str_lev1 = m_level1.group(4);
//			System.out.println(count + ":level1:"+str_lev1);
			/**
			 * Level2: <a href=
			 * "http://www.pocomatic.com/docs/whitepapers/webservices/" class=l
			 * onmousedown="return clk(0,'','','','2','','0CBgQFjAB')"
			 * >Component-Based <em>WebServices</em> and SCA in PocoCapsule -
			 * <em>Pocomatic</em> <b>...</b></a>
			 */
			//String s = "...<a href=\"http://www.pocomatic.com/docs/whitepapers/webservices/\" taget=_blank class=l onmousedown=\"return clk(0,'','','','2','','0CBgQFjAB')\">Component-Based <em>WebServices</em> and SCA in PocoCapsule - <em>Pocomatic</em> <b>...</b></a>...";
			Pattern pattern_lev2 = Pattern
					.compile(".+?<a href=\"(.+?)\" target=\"{0,1}_blank\"{0,1} class=.+? onmousedown=\"return clk.+?\">(.+?)</a>.+?",Pattern.DOTALL);
			Matcher m_level2 = pattern_lev2.matcher(str_lev1);
			if(!m_level2.matches()){
				pattern_lev2 = Pattern
				.compile(".+?<a href=\"(.+?)\" class=.+? onmousedown=\"return clk.+?\">(.+?)</a>.+?",Pattern.DOTALL);
				m_level2 = pattern_lev2.matcher(str_lev1);
			}
			if(!m_level2.matches()){
				pattern_lev2 = Pattern
				.compile(".+?<a href=\"(.+?)\" target=\"{0,1}_blank\"{0,1} class=\"{0,1}l\"{0,1}>(.+?)</a>.+?",Pattern.DOTALL);
				m_level2 = pattern_lev2.matcher(str_lev1);
			}
			if(!m_level2.matches()){
				pattern_lev2 = Pattern
				.compile(".+?<a href=\"(.+?)\" class=\"{0,1}l\"{0,1}>(.+?)</a>.+?",Pattern.DOTALL);
				m_level2 = pattern_lev2.matcher(str_lev1);
			}
			if (m_level2.matches()) {
				String backlinkURL = m_level2.group(1);
//				System.out.println("==========" + backlinkURL);
				
				/**
				 * Component-Based <em>WebServices</em> and SCA in PocoCapsule -
				 * <em>Pocomatic</em> <b>...</b>
				 */
				String anchorTxt_raw = m_level2.group(2);
//				System.out.println("++++++++++" + anchorTxt_raw);
				/*
				 * Remove html tags
				 */
				String anchorTxt = removeHTMLTags(anchorTxt_raw);

				item.setAnchorText(anchorTxt);
				item.setOriginalURL(backlinkURL);
				//System.out.println("level2:"+backlinkURL);
				//System.out.println("level3:"+anchorTxt);
			}else{
				System.out.println("------level2 error");
			}
			/**
			 * Level 3: <div class="s">&lt;binding.ws port=&quot; <em>http</em>
			 * ://www.<em>bigbank</em>.com/AccountService <b>...</b>
			 * &lt;reference name=&quot;<wbr> <em>StockQuoteWebService</em>
			 * &quot;&gt;. &lt;interface. <em>wsdl</em> interface=&quot;
			 * <em>http</em> ://www. <b>...</b> as illustrated in the calculator
			 * <em>example</em> in the <em>examples</em>/<em>web</em>-
			 * <em>services</em>/calculator-<wbr><em>ocsa</em> directory of
			 * <b>...</b> This is a basic client/server
			 * <em>Web Services example</em> deployed using <b>...</b><br>
			 * <span class=f><cite>www.<b>pocomatic.com/docs</b>/whitepapers/
			 * <b>webservices</b>/</cite> - <span class=gl><a href="http://webcache.googleusercontent.com/search?q=cache:CDYndBBl_EsJ:www.pocomatic.com/docs/whitepapers/webservices/+http://www.pocomatic.com/docs/cpp-examples/web-services/bigbank-ocsa/StockQuoteWebService.wsdl&amp;cd=2&amp;hl=en&amp;ct=clnk"
			 * onmousedown
			 * ="return clk(0,'','','','2','','0CBoQIDAB')">Cached</a> - <a
			 * href="/search?hl=en&amp;q=related:www.pocomatic.com/docs/whitepapers/webservices/+http://www.pocomatic.com/docs/cpp-examples/web-services/bigbank-ocsa/StockQuoteWebService.wsdl&amp;tbo=1&amp;sa=X&amp;ei=JXmVTPmRHZa0cOWBsaQF&amp;ved=0CBsQHzAB"
			 * >Similar</a></span></span></div>
			 */
			//String s2 = "<div class=\"s\">&lt;binding.ws port=&quot;<em>http</em>://www.<em>bigbank</em>.com/AccountService <b>...</b> &lt;reference name=&quot;<wbr><em>StockQuoteWebService</em>&quot;&gt;. &lt;interface.<em>wsdl</em> interface=&quot;<em>http</em>://www. <b>...</b> as illustrated  in the calculator <em>example</em> in the <em>examples</em>/<em>web</em>-<em>services</em>/calculator-<wbr><em>ocsa</em> directory of <b>...</b> This is a basic client/server <em>Web Services example</em> deployed using <b>...</b><br><span class=f><cite>www.<b>pocomatic.com/docs</b>/whitepapers/<b>webservices</b>/</cite> - <span class=gl><a href=\"http://webcache.googleusercontent.com/search?q=cache:CDYndBBl_EsJ:www.pocomatic.com/docs/whitepapers/webservices/+http://www.pocomatic.com/docs/cpp-examples/web-services/bigbank-ocsa/StockQuoteWebService.wsdl&amp;cd=2&amp;hl=en&amp;ct=clnk\" onmousedown=\"return clk(0,'','','','2','','0CBoQIDAB')\">Cached</a> - <a href=\"/search?hl=en&amp;q=related:www.pocomatic.com/docs/whitepapers/webservices/+http://www.pocomatic.com/docs/cpp-examples/web-services/bigbank-ocsa/StockQuoteWebService.wsdl&amp;tbo=1&amp;sa=X&amp;ei=JXmVTPmRHZa0cOWBsaQF&amp;ved=0CBsQHzAB\">Similar</a></span></span></div>";
			// String s2 = "<div class=\"s\">adfas<span class=f>";
			Pattern pattern_lev3 = Pattern
					.compile(".*<div class=\"s\">(.+?)<span class=f>.*");
			Matcher m_level3 = pattern_lev3.matcher(str_lev1);
			if (!m_level3.matches()){
				pattern_lev3 = Pattern
				.compile(".*<div class=\"s\">(.+?)<span class=\"f\">.*");
				m_level3 = pattern_lev3.matcher(str_lev1);
			}
			if (m_level3.matches()) {
				String summary_raw = m_level3.group(1);
				String summary = removeHTMLTags(summary_raw);
//				System.out.println(summary);
				item.setSummary(summary);
				lists.add(item);
			}
		}
		return lists;
	}
	public static int getResultNumOfQuery(String pageContent){
		int result = 0;
		//<div id=resultStats>找到约 485,000,000 条结果<nobr>  （用时 0.06 秒）&nbsp;</nobr></div>
		String str_pattern = "((<div id=resultStats>)|(<div id=\"resultStats\">)).+?</div>";
		Pattern pattern = Pattern.compile(str_pattern, Pattern.DOTALL|Pattern.CASE_INSENSITIVE);
		Matcher matcher = pattern.matcher(pageContent);
		
		if (matcher.find()){
			StringBuffer sb = new StringBuffer();
			boolean isBegin = false;
			for (int i=matcher.start(); i<matcher.end(); ++i){
				if (pageContent.charAt(i) <= '9' && pageContent.charAt(i) >= '0'){
					isBegin = true;
					sb.append(pageContent.charAt(i));
				}else if (pageContent.charAt(i) == ','){
				}else{
					if (isBegin == true){
						break;
					}
				}
			}
			try{
				result = Integer.parseInt(sb.toString());
			}catch (Exception e) {
				e.printStackTrace();
			}
		}
		return result;
	}

	private static String removeHTMLTags(String rawStr) {
		String[] raw = { "<.??em>", "<.??b>", "<br.??>", "&quot;", "&amp;",
				"&lt;", "&gt;" };
		String[] cle = { "", "", "", "\"", "&", "<", ">" };
		String newStr = rawStr;
		for (int i = 0; i < raw.length; i++) {
			newStr = newStr.replaceAll(raw[i], cle[i]);
		}
		return newStr;
	}
}
